The goals / steps of this project are the following:
from tqdm import tqdm
import numpy as np
import cv2
import glob
import matplotlib.pyplot as plt
from moviepy.editor import VideoFileClip
from IPython.display import HTML
%matplotlib inline
# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
# no magic numbers: define some constants
# # # # # # # # # # # # # # # # # # # # # # # # # #
# for calibration
_ROW, _COL, _NUMCHAN = 6, 9, 3
_image_size = (1280, 720)
objp = np.zeros((_ROW*_COL,_NUMCHAN), np.float32)
objp[:,:2] = np.mgrid[0:_COL,0:_ROW].T.reshape(-1,2)
# Credit to http://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_calib3d/py_calibration/py_calibration.html
# for the idea of refining the image points
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# Arrays to store object points and image points from all the images.
objpoints = [] # 3d points in real world space
imgpoints = [] # 2d points in image plane.
# Make a list of calibration images
images = glob.glob('../camera_cal/calibration*.jpg')
flag = True
cal = None
# Step through the list and search for chessboard corners
for fname in tqdm(images):
img = cv2.imread(fname)
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
# Find the chessboard corners
ret, corners = cv2.findChessboardCorners(gray,(_COL,_ROW),None)
# If found, add object points, image points
if ret == True:
objpoints.append(objp)
corners = cv2.cornerSubPix(gray,corners,(11,11),(-1,-1),criteria)
imgpoints.append(corners)
if(flag == True):
cal = cv2.drawChessboardCorners(img, (_COL,_ROW), corners, ret)
flag = False
else:
print("no corners found for ", fname) #The errors I get are presumably due to mismatching points in images
shape = cv2.imread(images[0]).shape[1::-1]
ret, matrix, dist_co, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, shape, None, None)
print("camera matrix C and distortion coefficients retrieved")
# Note that the camera matrix and distortion coefficients were defined once, which makes this straightforward
def undistort(image):
return cv2.undistort(image, matrix, dist_co, None, matrix)
# Let's check that it worked
test = cv2.imread('../camera_cal/calibration1.jpg')
undist = undistort(test)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(30,10))
f.tight_layout()
ax1.imshow(test)
ax1.set_title('Original Image')
ax2.imshow(undist)
ax2.set_title('Undistorted Image')
Looks good!
# Let's first take a look at an undistorted image to make sure our calibration was correct and
# get an idea for some source points to map to.
image = cv2.cvtColor(cv2.imread("../test_images/straight_lines1.jpg"), cv2.COLOR_BGR2RGB)
image2 = cv2.cvtColor(cv2.imread("../test_images/test6.jpg"), cv2.COLOR_BGR2RGB)
undist = undistort(image)
undist2 = undistort(image2)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(30,10))
ax1.set_title('Original Image', fontsize=20)
ax1.imshow(image)
ax2.set_title('Undistorted Image', fontsize=20)
ax2.imshow(undist)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(30,10))
ax1.set_title('Original Image', fontsize=20)
ax1.imshow(image2)
ax2.set_title('Undistorted Image', fontsize=20)
ax2.imshow(undist2)
Observe how the green sign on the righthand side (at the edge) of the first image is shifted slightly. The white Toyota on the right is also pulled to the right as well, and the tree on the left is also pulled closer. Seems good!
import Threshold
#thresh = Threshold.threshold(undist, do_b=True, do_l=True)
#thresh2 = Threshold.threshold(undist2, do_b=True, do_l=True)
thresh = Threshold.final_threshold(undist)
thresh2 = Threshold.final_threshold(undist2)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(30,10))
ax1.set_title('Thresh')
ax1.imshow(thresh)
ax2.set_title('Thresh2')
ax2.imshow(thresh2)
Jazzy! And pretty good actually - lines are still clearly visible.
# Hmm, from earlier it seems like (160, 715) and (1120, 715) are our bottom two points, (560, 465) and (730, 465)
# the upper points.
# How do we map to the rectangle in a perspective transform? Well we know our y-coordinates (min and max of image
# along y-axis), but let's try to find some decent x-coordinates. Let's try going for the quartiles and pad them.
# I tried the quartiles without modification, but I got some seriously weird curvatures. Padding fixes this, not
# entirely sure why.
x_left = image.shape[1] / 4 + 0
x_right = ((image.shape[1] / 4) * 3) - 0
y_min, y_max = 0, image.shape[0]
points = [[[560, 465]], [[730, 465]], [[1120, 715]], [[160, 715]]]
src = np.float32(points)
dst = np.float32([[x_left, y_min], [x_right, y_min], [x_right, y_max], [x_left, y_max]])
M = cv2.getPerspectiveTransform(src, dst)
Minv = cv2.getPerspectiveTransform(dst, src)
# note that the image needs to be undistorted!
def transform(image):
return cv2.warpPerspective(image, M, _image_size, flags=cv2.INTER_LINEAR)
bird = transform(undist)
bird2 = transform(undist2)
drawing = True
if(drawing):
temp_undist = np.copy(undist)
temp_undist2 = np.copy(undist2)
temp_bird = np.copy(bird)
temp_bird2 = np.copy(bird)
temp_bird_thresh = Threshold.final_threshold(bird)
temp_bird_thresh2 = Threshold.final_threshold(bird2)
pts = np.array([points[3],points[2],points[1],points[0]], np.int32)
pts = pts.reshape((-1,1,2))
cv2.polylines(temp_undist,[pts],True,(255,0,0), thickness=5)
pts = np.array([points[3], points[2], points[1], points[0]], np.int32)
pts = pts.reshape((-1,1,2))
cv2.polylines(temp_undist2,[pts],True,(255,0,0), thickness=5)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(30, 10))
ax1.imshow(temp_undist)
ax1.set_title('Undistorted Image', fontsize=40)
ax2.imshow(temp_bird_thresh)
ax2.set_title('Thresholded and Warped Image', fontsize=40)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(30,10))
ax1.imshow(temp_undist2)
ax1.set_title('Undistorted Image', fontsize=40)
ax2.imshow(temp_bird_thresh2)
ax2.set_title('Thresholded and Warped Image', fontsize=40)
Pulling the corners in a bit, but that's not bad and lines are looking clear as day!
import collections
import Lane
LEFT = Lane.LEFT
RIGHT = Lane.RIGHT
# note: these all accept coloured images
# as you can probably guess, these were testing functions (video was being odd with them until I realised
# I had to convert from RGB, which is seriously annoying). process_image is the final one used.
def no_change(image):
return undistort(image)
def dummy(image):
undist = undistort(image)
bird = transform(undist)
thresh = Threshold.final_threshold(bird)
img_bin = np.dstack((thresh*255, thresh*255, thresh*255))
return img_bin
def test_thresh(image):
undist = undistort(image)
plt.imshow(undist)
plt.show()
thresh = Threshold.final_threshold(undist)
plt.imshow(thresh)
plt.show()
def process_image_test_warp(image):
undist = undistort(image)
thresh = Threshold.final_threshold(undist)
bird = transform(thresh)
return bird
def process_image(image):
undist = undistort(image)
bird = transform(undist)
bird_thresh = Threshold.final_threshold(bird)
display = Lane.video_pipeline(undist, bird_thresh, Minv)
return display
def display_lines(image):
undist = undistort(image)
bird = transform(undist)
thresh = Threshold.final_threshold(bird)
display = Lane.display_lines(thresh)
return display
def get_lines(image):
undist = undistort(image)
thresh = Threshold.final_threshold(undist)
bird = transform(thresh)
display = Lane.get_lines(bird)
return display
images = glob.glob('../test_images/*.jpg')
image = cv2.cvtColor(cv2.imread(images[4]), cv2.COLOR_BGR2RGB)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(30, 10))
ax1.imshow(undistort(image))
ax1.set_title('Undistorted Image', fontsize=40)
ax2.imshow(dummy(image))
ax2.set_title('display_lines', fontsize=40)
for i in images:
image = cv2.cvtColor(cv2.imread(i), cv2.COLOR_BGR2RGB)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(30, 10))
ax1.imshow(undistort(image))
ax1.set_title('Undistorted Image', fontsize=40)
ax2.set_title('Searched Lines', fontsize=40)
ax2.imshow(display_lines(image))
setting = True
if setting:
image = cv2.cvtColor(cv2.imread(images[4]), cv2.COLOR_BGR2RGB)
undist = undistort(image)
bird = process_image_test_warp(image)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(30, 10))
ax1.imshow(undist)
ax1.set_title('Undistorted Image', fontsize=40)
result = Lane.get_radius(undist, bird, Minv)
ax2.imshow(result)
ax2.set_title('Radius-derived lines', fontsize=40)
setting = True
if setting:
for i in images:
image = cv2.cvtColor(cv2.imread(i), cv2.COLOR_BGR2RGB)
undist = undistort(image)
bird = process_image_test_warp(image)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(30, 10))
ax1.imshow(undist)
ax1.set_title('Undistorted Image', fontsize=40)
result = Lane.get_radius(undist, bird, Minv)
ax2.imshow(result)
ax2.set_title('Radius-derived lines', fontsize=40)
Lane.LEFT.reset()
Lane.RIGHT.reset()
## Video processing, etc.
output = '../output_video/output_video.mp4'
clip1 = VideoFileClip("../videos/project_video.mp4")
video_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!
%time video_clip.write_videofile(output, audio=False)
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(output))